const puppeteer = require('puppeteer');
const fs = require('fs');

let totalBlogs = [];
let errLogs = [];

async function main() {
  let tab = 'weekly';
  let sub_tab = 'FEX技术周刊';
  const fileName = `fex.json`;
  let totalBlogs = [];
  let blogs = [];

  const browser = await puppeteer.launch({headless: true, slowMo: 0});
  const page = (await browser.pages())[0];
  await page.setViewport({width: 1500, height: 0});

  page.on('pageerror', msg => {
    console.log(msg);
  });
  page.on('console', msg => {
    console.log(msg.text());
  });

  let url = `http://fex.baidu.com/weekly/`;
  await page.goto(url, {waitUntil: 'domcontentloaded'});

  let arr = await page.$$eval('body > div.container > div.articles > ul > li', async (eles) => {
    let result = [];
    eles.forEach(async ele => {
      let url = ele.querySelector('a').href;
      result.push({url: url});
    });
    return result;
  });

  for (let i = 0; i < arr.length; i++) {
    try {
      let item = arr[i];
      await page.goto(item['url'], {waitUntil: 'domcontentloaded'});
      await page.waitFor(1000);
      let create_at = await page.$eval('body > div.container > article > div.article-info > h1', el => el.innerText);
      create_at = create_at.replace(/.*?(\d+)\/(\d+)\/(\d+)/, '$1-$2-$3');
      blogs = await page.$$eval('body > div.container > article > div.content > p', async (eles, create_at) => {
        let tmpBlogs = [];
        eles.forEach(async ele => {
          let title = ele.querySelector('strong').innerText;
          let content = ele.querySelector('a').href;
          let intro = ele.innerHTML.split('<br>')[ele.innerHTML.split('<br>').length - 1];
          if (intro) {
            intro = intro.trim();
          }
          if (/.*[\u4e00-\u9fa5]+.*/.test(title)) {
            tmpBlogs.push({
              title: title,
              content: content,
              original_url: content,
              tab: 'weekly',
              sub_tab: 'FEX技术周刊',
              source: 'FEX技术周刊',
              intro: intro,
              create_at: create_at
            });
          }
        });
        return tmpBlogs;
      }, create_at);
      console.log(i, arr.length, blogs.length);
      totalBlogs = totalBlogs.concat(blogs);
    } catch (e) {
      console.log(e.message);
      continue;
    }
  }


  let writerStream = fs.createWriteStream(`crawler_data/${fileName}`);
  writerStream.write(JSON.stringify(totalBlogs));
  console.log('write done');

};


async function collectBlogs(page, tab, subTab) {
  let blogs = await page.$$eval('body > div.Central_wrap > div > div.Central_left.left > div > ul > li', async (eles, tab, sub_tab) => {
    let tmpBlogs = [];
    eles.forEach(async ele => {
      let title = ele.querySelector('div.rinfo > a').innerText;
      let content = ele.querySelector('div.rinfo > a').href;
      let intro = ele.querySelector('div.rinfo > p').innerText;
      let blog_date = ele.querySelector('div.rinfo > div > i').innerText;
      tmpBlogs.push({
        title: title,
        content: content,
        tab: tab,
        sub_tab: sub_tab,
        blog_date: blog_date
      });
    });
    return tmpBlogs;
  }, tab, subTab);
  return blogs;
}

main();


